require 'curb'
require 'rexml/document'
require 'html/xmltree'
require 'crawler/crawler'

module Crawler
  
  class StumbleuponCrawler < Crawler
    
    public
    
    def initialize(options = {})
      super options
      @parser = HTMLTree::XMLParser.new false, true
      @curl.follow_location = true
      @curl.max_redirects = -1
    end
    
    def crawl_table
      StumbleuponCrawl
    end
    
    def service_name
      "stumbleupon"
    end
    
    protected
    
    def crawl(c)
      @curl.url = "http://www.stumbleupon.com/stumbler/#{c.service.username_on_service}/"
      
      @curl.http_get
      @parser.reset
      @parser.feed @curl.body_str
      r = REXML::XPath.match @parser.document.root, './/title/text() = "Error: No Such Username - StumbleUpon"'
      
      if !r[0] then
        c.body =  @curl.body_str
        c.status =  @curl.response_code
        #c.crawled_at = DateTime.now
      else
        # the user does not exist
        c.status = 404
        c.body = nil
      end
      c.status == 200
    end
    
    def has_crawl?(s)
      !s.stumbleupon_crawl.nil?
    end
  end
  
end # module Crawler